#!/usr/bin/python3
# -*- coding: utf-8; tab-width: 4; indent-tabs-mode: t -*-

import os
import re
import sys
import time
import mrget
import functools
import subprocess
import mirrors.plugin
from datetime import datetime


class Main:

    def __init__(self):
        self.id = mirrors.plugin.params["id"]
        self.cfg = mirrors.plugin.params["config"]
        self.dataDir = mirrors.plugin.params["storage-file"]["data-directory"]

        # download source
        self.fileUrlList = []

    def run(self):
        # get torrent file page
        root = None

        # process
        fileListDict = dict()
        for dataName, dataFunc in self.DATA_DICT.items():
            if dataName not in self.cfg:
                continue

            print("Processing %s..." % (dataName))
            dataFunc(root)



        # self.rsyncUrl
        if True:
            filter_key = lambda x: x["protocol"] == "rsync" and x["role"] != "local-mirror"
            urlList = mrget.target_urls("mirror://kiwix", filter_key=filter_key)
            if len(urlList) == 0:
                print("no appropriate rsync source found", file=sys.stderr)
                sys.exit(1)
            self.rsyncUrl = urlList[0]

        # self.fileUrlList
        if True:
            filter_key = lambda x: x["protocol"] in ["http", "https", "ftp"] and x["role"] != "local-mirror"
            self.fileUrlList = mrget.target_urls("mirror://kiwix", filter_key=filter_key)
            if len(self.fileUrlList) == 0:
                print("no appropriate file source found", file=sys.stderr)
                sys.exit(1)

        # print self.rsyncUrl and self.fileUrlList
        print("Found:")
        print("    rsync source: %s" % (self.rsyncUrl))
        for url in self.fileUrlList:
            print("    file source: %s" % (url))

        # download
        fileListDict = dict()
        for dataName, filterFunc in self.DATA_DICT.items():
            print("Synchronizing %s..." % dataName)

            zimPath = os.path.join("zim", dataName)
            zimLocalDir = os.path.join(self.dataDir, zimPath)

            fileList = self._rsyncGetFileList(zimPath)
            fileList = filterFunc(self.cfg.get(dataName, {}), fileList)

            os.makedirs(zimLocalDir, exist_ok=True)
            self._rsync(zimPath, fileList, zimLocalDir)

            fileListDict[zimLocalDir] = fileList

        # delete
        for zimLocalDir, fileList in fileListDict.items():
            for fn in os.listdir(zimLocalDir):
                if fn not in fileList:
                    os.unlink(os.path.join(zimLocalDir, fn))

    def _rsyncGetFileList(self, zimPath):
        ret = []
        while True:
            try:
                url = os.path.join(self.rsyncUrl, zimPath)
                out = subprocess.check_output(["rsync", "--no-motd", "--list-only", "%s/*" % (url)], universal_newlines=True)    # always re-try
                for line in out.split("\n"):
                    # "drwxr-xr-x          2,380 2021/01/18 12:55:02 ./abc" -> "./abc"
                    m = re.fullmatch(r'\S+ +\S+ +\S+ +\S+ +(.*)', line)
                    if m is None:
                        continue
                    if m.group(1) == ".":
                        continue
                    ret.append(m.group(1))
                return ret
            except subprocess.CalledProcessError as e:
                print("rsync listing files failed and try again: %s" % str(e))
                time.sleep(60)

    def _rsync(self, zimPath, fileList, zimLocalDir):
        # we use "-rlptD" insead of "-a" so that the remote user/group is ignored
        cmdList = ["rsync", "-rlptD", "-z", "-v", "--partial", "-H"]
        for fn in fileList:
            cmdList += ["-f", "+ %s" % (fn)]
        cmdList += ["-f", "- *"]
        cmdList += [os.path.join(self.rsyncUrl, zimPath, "*"), zimLocalDir]
        print(cmdList)

        # do work
        while True:
            try:
                subprocess.check_call(cmdList)      # always re-try
                break
            except subprocess.CalledProcessError as e:
                print("rsync failed and try again: %s" % str(e))
                time.sleep(60)

    def _filterKiwixSimple(self, bTrivial, cfg, fileList):
        # "include-lang", "exclude-lang" in config:
        langIncList, langExcList = _LangFilter.getLangIncExcList(cfg)

        ret = []
        for fn in fileList:
            if not bTrivial:
                # example: XXXXXXXX_ab_all_2020-11.zim
                # we only download "_all_" category files
                m = re.fullmatch(r"\S+_(\S+)_all_\S+\.zim", fn)
            else:
                # example: XXXXXXXX_ab_2020-11.zim
                m = re.fullmatch(r"\S+_(\S+)_\S+\.zim", fn)
            if m is None:
                continue
            if _LangFilter.testByLang(langIncList, langExcList, m.group(1)):        # filter by "include-lang", "exclude-lang"
                continue
            ret.append(fn)

        # filter by date
        ret = _DateFilter.filterByDate(ret)

        return ret

    def _filterKiwixTed(self, cfg, fileList):
        # "include-lang", "exclude-lang" in config:
        langIncList, langExcList = _LangFilter.getLangIncExcList(cfg)

        ret = []
        for fn in fileList:
            # example: ted_en_BLABLABLA_2020-11.zim
            m = re.fullmatch(r"\S+_(\S+)_\S+_\S+\.zim", fn)
            if m is None:
                continue
            if _LangFilter.testByLang(langIncList, langExcList, m.group(1)):        # filter by "include-lang", "exclude-lang"
                continue
            ret.append(fn)

        # filter by date
        ret = _DateFilter.filterByDate(ret)

        return ret

    def _filterKiwixVideos(self, cfg, fileList):
        # "include-lang", "exclude-lang" in config:
        langIncList, langExcList = _LangFilter.getLangIncExcList(cfg)

        ret = []
        for fn in fileList:
            # example (with language):    khan-academy-videos_tr_organik-kimya-fen-bilimleri_2020-12.zim
            # example (without language): voa_learning_english-eim-english-in-a-minute_2021-11.zim
            m = re.fullmatch(r"(\S+)_(\S+)_\S+_\S+\.zim", fn)
            if m is None:
                continue
            if m.group(1) == "voa":
                ret.append(fn)
            else:
                if _LangFilter.testByLang(langIncList, langExcList, m.group(2)):    # filter by "include-lang", "exclude-lang"
                    continue
                ret.append(fn)

        # filter by date
        ret = _DateFilter.filterByDate(ret)

        return ret

    def _filterKiwixMediaWiki(self, bTrivial, cfg, fileList):
        # "file-type" in config:
        fileType = _FileTypeFilter.getFileType(cfg)
        # "include-lang", "exclude-lang" in config:
        langIncList, langExcList = _LangFilter.getLangIncExcList(cfg)

        ret = []
        for fn in fileList:
            if not bTrivial:
                # example: XXXXX_ru_all_maxi_2022-07.zim
                # we only download "_all_" category files
                m = re.fullmatch(r"\S+_(\S+)_all_(%s)_\S+\.zim" % (_FileTypeFilter.getFileTypeRegStr()), fn)
            else:
                # example: XXXXX_ru_maxi_2022-07.zim
                m = re.fullmatch(r"\S+_(\S+)_(%s)_\S+\.zim" % (_FileTypeFilter.getFileTypeRegStr()), fn)
            if m is None:
                continue
            if _FileTypeFilter.testByFileType(fileType, m.group(2)):            # filter by "file-type"
                continue
            if _LangFilter.testByLang(langIncList, langExcList, m.group(1)):    # filter by "include-lang", "exclude-lang"
                continue
            ret.append(fn)

        # filter by date
        ret = _DateFilter.filterByDate(ret)

        return ret


class _LangFilter:

    @staticmethod
    def getLangIncExcList(cfg):
        #   wikipedia_ab_all_maxi_2020-11.zim
        #             ^^

        langIncList = []
        langExcList = []
        if "include-lang" in cfg:
            langIncList = cfg["include-lang"]
        if "exclude-lang" in cfg:
            langExcList = cfg["exclude-lang"]
        if len(langIncList) > 0 and len(langExcList) > 0:
            raise Exception("\"include-lang\" and \"exclude-lang\" can not co-exist in config")
        return langIncList, langExcList

    @staticmethod
    def testByLang(langIncList, langExcList, lang):
        if len(langIncList) > 0:
            if lang not in langIncList:
                return True
        else:
            if lang in langExcList:
                return True
        return False


class scummvm-collection:

    @staticmethod
    def update(oldDir, newDir, updateDir):
        pass

    @staticmethod
    def checkDirByFullTorrentFile(path, torrentFile):
        pass


###############################################################################

if __name__ == "__main__":
    Main().run()



















#!/usr/bin/python3
# -*- coding: utf-8; tab-width: 4; indent-tabs-mode: t -*-

import mrget
import subprocess
import mirrors.plugin


def main():
    id = mirrors.plugin.params["id"]
    dataDir = mirrors.plugin.params["storage-file"]["data-directory"]




https://www.progettosnaps.net/dats/scummvm-collection
https://www.progettosnaps.net/samples/


    def filterFunc(x):
        return x["protocol"] == "rsync" and x["role"] != "local-mirror"

    if id == "centos":
        rsyncSource = mrget.target_urls("mirror://centos", filter_key=filterFunc)[0]
    elif id == "centos-altarch":
        rsyncSource = mrget.target_urls("mirror://centos-altarch", filter_key=filterFunc)[0]
    elif id == "centos-vault":
        rsyncSource = mrget.target_urls("mirror://centos-vault", filter_key=filterFunc)[0]
    else:
        assert False

    subprocess.run(["rsync", "-v", "-a", "-z", "-H", "--delete", rsyncSource, dataDir])


if __name__ == "__main__":
    main()
